{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled0.ipynb",
      "version": "0.3.2",
      "views": {},
      "default_view": {},
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "AFIchtHPxrM5",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "import tensorflow as tf\n",
        "from __future__ import absolute_import\n",
        "from __future__ import division\n",
        "from __future__ import print_function\n",
        "import argparse\n",
        "import sys\n",
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "import urllib\n",
        "import time\n",
        "FLAGS = None"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1aYBNpD-zglv",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 559
        },
        "outputId": "c2b8b164-2c87-4203-9afe-63c7411af09f",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1529491404655,
          "user_tz": -480,
          "elapsed": 1731,
          "user": {
            "displayName": "Lip Gallagher",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "113091702821929511633"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# load data\n",
        "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
        "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-2-7e828717a4ff>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please write your own downloading logic.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use urllib or similar directly.\n",
            "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting /tmp/tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
            "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting /tmp/tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.one_hot on tensors.\n",
            "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
            "Extracting /tmp/tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
            "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
            "Extracting /tmp/tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "FjT1WThmzka2",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def weight_variable(shape):\n",
        "    initial = tf.truncated_normal(shape, stddev=0.01) #权重\n",
        "    return tf.Variable(initial, collections=[tf.GraphKeys.GLOBAL_VARIABLES,'Weights'])\n",
        "def bias_variable(shape):\n",
        "    initial = tf.truncated_normal(shape, stddev=0.001)\n",
        "    return tf.Variable(initial)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FwguWk_zznsZ",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "x = tf.placeholder(tf.float32, [None, 784])#\n",
        "y_ = tf.placeholder(tf.float32, [None, 10])#\n",
        "keep_prob = tf.placeholder(tf.float32) #dropout"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bM-jS-lCzp-k",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.name_scope('reshape'):\n",
        "    x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
        "\n",
        "with tf.name_scope('conv1'):\n",
        "    w_conv1 = weight_variable([5,5,1,32])\n",
        "    b_conv1 = bias_variable([32])\n",
        "    l_conv1 = tf.nn.conv2d(x_image, w_conv1, strides=[1,1,1,1],\n",
        "                          padding = 'SAME') + b_conv1\n",
        "    # 激活函数\n",
        "    h_conv1 = tf.nn.relu(l_conv1)\n",
        "    #h_conv1_drop = tf.nn.dropout(h_conv1,keep_prob)\n",
        "#output batch,28,28,32"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jBJMAAk8zr-A",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# 28*28 - 14*14(32)\n",
        "with tf.name_scope('pool1'):\n",
        "    h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1,2,2,1],\n",
        "                            strides =[1,2,2,1], padding='SAME')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FSGuftHAzuA6",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.name_scope('conv2'):\n",
        "    w_conv2 = weight_variable([5,5,32,64])\n",
        "    b_conv2 = bias_variable([64])\n",
        "    l_conv2 = tf.nn.conv2d(h_pool1, w_conv2, strides=[1,1,1,1],\n",
        "                          padding='SAME') + b_conv2\n",
        "    h_conv2 = tf.nn.relu(l_conv2)\n",
        "    #h_conv2_drop = tf.nn.dropout(h_conv2)\n",
        "# output batch,14,14,64"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ebphhYCszvtS",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "h_conv2_drop = tf.nn.dropout(h_conv2, keep_prob) # conv2后使用dropout"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rEJLfOPbzxq-",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.name_scope('pool2'):\n",
        "    h_pool2 = tf.nn.max_pool(h_conv2_drop, ksize=[1,2,2,1],\n",
        "                            strides=[1,2,2,1], padding='SAME')\n",
        "# 14x14x64 -->  7x7x64"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hbZIZGdj-Qqz",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "#with tf.name_scope('conv3'):\n",
        "#  w_conv3 = weight_variable([4,4,12,24])\n",
        "#  b_conv3 = bias_variable([24])\n",
        "#  l_conv3 = tf.nn.conv2d(h_conv2, w_conv3, strides=[1,2,2,1],\n",
        "#                        padding='SAME') + b_conv3\n",
        "#  h_conv3 = tf.nn.relu(l_conv3)\n",
        "# output batch,7,7,24"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Plh29ot5zzwu",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.name_scope('fc1'):\n",
        "    w_fc1 = weight_variable([7*7*64, 1024])\n",
        "    b_fc1 = bias_variable([1024])\n",
        "    h_pool2_flat = tf.reshape(h_pool2, [-1,7*7*64])\n",
        "    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wB9OGzioz5dt",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "v7YFjNdsz7WB",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.name_scope('fc2'):\n",
        "    w_fc2 = weight_variable([1024,10])\n",
        "    b_fc2 = bias_variable([10])\n",
        "    y = (tf.matmul(h_fc1_drop, w_fc2) + b_fc2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "d0YaC8aAz82C",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "decay_rate = 0.96\n",
        "\n",
        "decay_steps = 1000\n",
        "\n",
        "global_ = tf.Variable(tf.constant(0))\n",
        "\n",
        "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y))\n",
        "\n",
        "#l2_loss = tf.add_n([tf.nn.l2_loss(w) for w in tf.get_collection('Weights')])\n",
        "\n",
        "#total_loss = cross_entropy + 7e-5*l2_loss\n",
        "\n",
        "\n",
        "\n",
        "#updateparameter = tf.group(update_parameter, update_parameter2)\n",
        "\n",
        "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_, 1))\n",
        "\n",
        "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
        "\n",
        "lr = tf.train.exponential_decay(0.001, global_, 1000,0.96 ) # (learning_rate, global, decay_step, decay_rate)\n",
        "train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KTSpS4qXz-rm",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 3635
        },
        "outputId": "794268f2-fed3-4986-f6e3-983ef742c956",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1529497192574,
          "user_tz": -480,
          "elapsed": 5776258,
          "user": {
            "displayName": "Lip Gallagher",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "113091702821929511633"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "with tf.Session() as sess:\n",
        "    init_op = tf.global_variables_initializer()\n",
        "    sess.run(init_op)\n",
        "    for step in range(20000):\n",
        "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
        "        ts, loss, l, = sess.run([train_step, cross_entropy, lr],\n",
        "                                 feed_dict={x:batch_xs, y_:batch_ys, keep_prob: 0.75, global_: step})\n",
        "        test_acc = sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
        "                                            y_: mnist.test.labels, keep_prob:1.0})\n",
        "        if (step+1) % 100==0:\n",
        "            print('step %d : entropy loss: %g, learning_rate: %g' %(step+1, loss, l))\n",
        "    print('test accuracy %g' % (test_acc))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "step 100 : entropy loss: 0.26826, learning_rate: 0.000995967\n",
            "step 200 : entropy loss: 0.108633, learning_rate: 0.000991909\n",
            "step 300 : entropy loss: 0.0844108, learning_rate: 0.000987869\n",
            "step 400 : entropy loss: 0.122211, learning_rate: 0.000983844\n",
            "step 500 : entropy loss: 0.167114, learning_rate: 0.000979836\n",
            "step 600 : entropy loss: 0.018806, learning_rate: 0.000975844\n",
            "step 700 : entropy loss: 0.105835, learning_rate: 0.000971869\n",
            "step 800 : entropy loss: 0.0879532, learning_rate: 0.000967909\n",
            "step 900 : entropy loss: 0.0149586, learning_rate: 0.000963966\n",
            "step 1000 : entropy loss: 0.042682, learning_rate: 0.000960039\n",
            "step 1100 : entropy loss: 0.0326038, learning_rate: 0.000956128\n",
            "step 1200 : entropy loss: 0.0752362, learning_rate: 0.000952233\n",
            "step 1300 : entropy loss: 0.0501056, learning_rate: 0.000948354\n",
            "step 1400 : entropy loss: 0.0452342, learning_rate: 0.00094449\n",
            "step 1500 : entropy loss: 0.021735, learning_rate: 0.000940642\n",
            "step 1600 : entropy loss: 0.0245398, learning_rate: 0.00093681\n",
            "step 1700 : entropy loss: 0.0138305, learning_rate: 0.000932994\n",
            "step 1800 : entropy loss: 0.0421491, learning_rate: 0.000929193\n",
            "step 1900 : entropy loss: 0.0446689, learning_rate: 0.000925408\n",
            "step 2000 : entropy loss: 0.034389, learning_rate: 0.000921638\n",
            "step 2100 : entropy loss: 0.0150812, learning_rate: 0.000917883\n",
            "step 2200 : entropy loss: 0.03383, learning_rate: 0.000914144\n",
            "step 2300 : entropy loss: 0.0948362, learning_rate: 0.00091042\n",
            "step 2400 : entropy loss: 0.0231628, learning_rate: 0.000906711\n",
            "step 2500 : entropy loss: 0.0108374, learning_rate: 0.000903017\n",
            "step 2600 : entropy loss: 0.00431046, learning_rate: 0.000899338\n",
            "step 2700 : entropy loss: 0.0033251, learning_rate: 0.000895674\n",
            "step 2800 : entropy loss: 0.00888547, learning_rate: 0.000892025\n",
            "step 2900 : entropy loss: 0.0208298, learning_rate: 0.000888391\n",
            "step 3000 : entropy loss: 0.00325081, learning_rate: 0.000884772\n",
            "step 3100 : entropy loss: 0.00175275, learning_rate: 0.000881168\n",
            "step 3200 : entropy loss: 0.00575854, learning_rate: 0.000877578\n",
            "step 3300 : entropy loss: 0.00803953, learning_rate: 0.000874003\n",
            "step 3400 : entropy loss: 0.00290522, learning_rate: 0.000870442\n",
            "step 3500 : entropy loss: 0.00403823, learning_rate: 0.000866896\n",
            "step 3600 : entropy loss: 0.0216621, learning_rate: 0.000863364\n",
            "step 3700 : entropy loss: 0.0161994, learning_rate: 0.000859847\n",
            "step 3800 : entropy loss: 0.00839407, learning_rate: 0.000856344\n",
            "step 3900 : entropy loss: 0.00192514, learning_rate: 0.000852856\n",
            "step 4000 : entropy loss: 0.00339, learning_rate: 0.000849381\n",
            "step 4100 : entropy loss: 0.00244225, learning_rate: 0.000845921\n",
            "step 4200 : entropy loss: 0.0103887, learning_rate: 0.000842475\n",
            "step 4300 : entropy loss: 0.00406988, learning_rate: 0.000839043\n",
            "step 4400 : entropy loss: 0.00247004, learning_rate: 0.000835624\n",
            "step 4500 : entropy loss: 0.0021554, learning_rate: 0.00083222\n",
            "step 4600 : entropy loss: 0.00176306, learning_rate: 0.00082883\n",
            "step 4700 : entropy loss: 0.00974655, learning_rate: 0.000825453\n",
            "step 4800 : entropy loss: 0.0187838, learning_rate: 0.00082209\n",
            "step 4900 : entropy loss: 0.00073611, learning_rate: 0.000818741\n",
            "step 5000 : entropy loss: 0.000373865, learning_rate: 0.000815406\n",
            "step 5100 : entropy loss: 0.00779213, learning_rate: 0.000812084\n",
            "step 5200 : entropy loss: 0.00356575, learning_rate: 0.000808776\n",
            "step 5300 : entropy loss: 0.00757275, learning_rate: 0.000805481\n",
            "step 5400 : entropy loss: 0.00136803, learning_rate: 0.000802199\n",
            "step 5500 : entropy loss: 0.0299129, learning_rate: 0.000798931\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "step 5600 : entropy loss: 0.00155505, learning_rate: 0.000795677\n",
            "step 5700 : entropy loss: 0.00358994, learning_rate: 0.000792435\n",
            "step 5800 : entropy loss: 0.00036983, learning_rate: 0.000789207\n",
            "step 5900 : entropy loss: 0.00099386, learning_rate: 0.000785992\n",
            "step 6000 : entropy loss: 6.77559e-05, learning_rate: 0.00078279\n",
            "step 6100 : entropy loss: 0.0013651, learning_rate: 0.000779601\n",
            "step 6200 : entropy loss: 0.000351457, learning_rate: 0.000776425\n",
            "step 6300 : entropy loss: 4.80618e-05, learning_rate: 0.000773262\n",
            "step 6400 : entropy loss: 0.00110854, learning_rate: 0.000770111\n",
            "step 6500 : entropy loss: 0.000906251, learning_rate: 0.000766974\n",
            "step 6600 : entropy loss: 0.0481559, learning_rate: 0.000763849\n",
            "step 6700 : entropy loss: 0.00400661, learning_rate: 0.000760738\n",
            "step 6800 : entropy loss: 6.24552e-05, learning_rate: 0.000757638\n",
            "step 6900 : entropy loss: 0.000209449, learning_rate: 0.000754552\n",
            "step 7000 : entropy loss: 0.00347291, learning_rate: 0.000751478\n",
            "step 7100 : entropy loss: 0.00370832, learning_rate: 0.000748417\n",
            "step 7200 : entropy loss: 1.85804e-05, learning_rate: 0.000745368\n",
            "step 7300 : entropy loss: 0.000439323, learning_rate: 0.000742331\n",
            "step 7400 : entropy loss: 0.000731393, learning_rate: 0.000739307\n",
            "step 7500 : entropy loss: 0.0123227, learning_rate: 0.000736295\n",
            "step 7600 : entropy loss: 0.000435027, learning_rate: 0.000733296\n",
            "step 7700 : entropy loss: 0.0695543, learning_rate: 0.000730308\n",
            "step 7800 : entropy loss: 0.000251712, learning_rate: 0.000727333\n",
            "step 7900 : entropy loss: 2.32289e-05, learning_rate: 0.00072437\n",
            "step 8000 : entropy loss: 0.0654284, learning_rate: 0.000721419\n",
            "step 8100 : entropy loss: 0.00470577, learning_rate: 0.00071848\n",
            "step 8200 : entropy loss: 0.00504927, learning_rate: 0.000715553\n",
            "step 8300 : entropy loss: 6.09591e-05, learning_rate: 0.000712638\n",
            "step 8400 : entropy loss: 0.0004942, learning_rate: 0.000709735\n",
            "step 8500 : entropy loss: 0.000303874, learning_rate: 0.000706843\n",
            "step 8600 : entropy loss: 0.00568765, learning_rate: 0.000703964\n",
            "step 8700 : entropy loss: 0.000333792, learning_rate: 0.000701096\n",
            "step 8800 : entropy loss: 0.000619556, learning_rate: 0.00069824\n",
            "step 8900 : entropy loss: 3.12387e-05, learning_rate: 0.000695395\n",
            "step 9000 : entropy loss: 0.000345104, learning_rate: 0.000692562\n",
            "step 9100 : entropy loss: 0.000587956, learning_rate: 0.000689741\n",
            "step 9200 : entropy loss: 3.04743e-05, learning_rate: 0.000686931\n",
            "step 9300 : entropy loss: 0.00483386, learning_rate: 0.000684132\n",
            "step 9400 : entropy loss: 0.00388445, learning_rate: 0.000681345\n",
            "step 9500 : entropy loss: 0.0017795, learning_rate: 0.00067857\n",
            "step 9600 : entropy loss: 0.000254304, learning_rate: 0.000675805\n",
            "step 9700 : entropy loss: 0.000306057, learning_rate: 0.000673052\n",
            "step 9800 : entropy loss: 0.000334637, learning_rate: 0.00067031\n",
            "step 9900 : entropy loss: 0.000313881, learning_rate: 0.000667579\n",
            "step 10000 : entropy loss: 0.00113795, learning_rate: 0.00066486\n",
            "step 10100 : entropy loss: 0.0365735, learning_rate: 0.000662151\n",
            "step 10200 : entropy loss: 0.0220135, learning_rate: 0.000659454\n",
            "step 10300 : entropy loss: 0.000131807, learning_rate: 0.000656767\n",
            "step 10400 : entropy loss: 0.00242316, learning_rate: 0.000654091\n",
            "step 10500 : entropy loss: 0.00474886, learning_rate: 0.000651427\n",
            "step 10600 : entropy loss: 4.8115e-05, learning_rate: 0.000648773\n",
            "step 10700 : entropy loss: 6.45544e-05, learning_rate: 0.00064613\n",
            "step 10800 : entropy loss: 0.00138597, learning_rate: 0.000643498\n",
            "step 10900 : entropy loss: 0.00844589, learning_rate: 0.000640876\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "step 11000 : entropy loss: 7.09919e-05, learning_rate: 0.000638265\n",
            "step 11100 : entropy loss: 0.0749492, learning_rate: 0.000635665\n",
            "step 11200 : entropy loss: 0.000259157, learning_rate: 0.000633075\n",
            "step 11300 : entropy loss: 0.0711535, learning_rate: 0.000630496\n",
            "step 11400 : entropy loss: 9.80535e-06, learning_rate: 0.000627928\n",
            "step 11500 : entropy loss: 0.000286971, learning_rate: 0.00062537\n",
            "step 11600 : entropy loss: 0.00367719, learning_rate: 0.000622822\n",
            "step 11700 : entropy loss: 3.75268e-05, learning_rate: 0.000620285\n",
            "step 11800 : entropy loss: 4.21644e-05, learning_rate: 0.000617758\n",
            "step 11900 : entropy loss: 0.00586243, learning_rate: 0.000615241\n",
            "step 12000 : entropy loss: 4.19535e-05, learning_rate: 0.000612735\n",
            "step 12100 : entropy loss: 1.89197e-05, learning_rate: 0.000610238\n",
            "step 12200 : entropy loss: 0.0193832, learning_rate: 0.000607752\n",
            "step 12300 : entropy loss: 0.0648841, learning_rate: 0.000605276\n",
            "step 12400 : entropy loss: 0.00111589, learning_rate: 0.000602811\n",
            "step 12500 : entropy loss: 2.68259e-05, learning_rate: 0.000600355\n",
            "step 12600 : entropy loss: 1.96707e-05, learning_rate: 0.000597909\n",
            "step 12700 : entropy loss: 6.64051e-06, learning_rate: 0.000595473\n",
            "step 12800 : entropy loss: 1.04757e-05, learning_rate: 0.000593047\n",
            "step 12900 : entropy loss: 5.60625e-05, learning_rate: 0.000590631\n",
            "step 13000 : entropy loss: 6.1677e-05, learning_rate: 0.000588225\n",
            "step 13100 : entropy loss: 1.6498e-06, learning_rate: 0.000585829\n",
            "step 13200 : entropy loss: 1.86883e-05, learning_rate: 0.000583442\n",
            "step 13300 : entropy loss: 2.63084e-06, learning_rate: 0.000581065\n",
            "step 13400 : entropy loss: 8.12506e-06, learning_rate: 0.000578698\n",
            "step 13500 : entropy loss: 0.00104189, learning_rate: 0.000576341\n",
            "step 13600 : entropy loss: 5.59323e-05, learning_rate: 0.000573993\n",
            "step 13700 : entropy loss: 1.21351e-05, learning_rate: 0.000571654\n",
            "step 13800 : entropy loss: 4.3616e-06, learning_rate: 0.000569325\n",
            "step 13900 : entropy loss: 1.29549e-05, learning_rate: 0.000567006\n",
            "step 14000 : entropy loss: 0.000322557, learning_rate: 0.000564696\n",
            "step 14100 : entropy loss: 4.04826e-05, learning_rate: 0.000562396\n",
            "step 14200 : entropy loss: 5.40902e-06, learning_rate: 0.000560105\n",
            "step 14300 : entropy loss: 3.11947e-06, learning_rate: 0.000557823\n",
            "step 14400 : entropy loss: 0.0569319, learning_rate: 0.00055555\n",
            "step 14500 : entropy loss: 1.98708e-06, learning_rate: 0.000553287\n",
            "step 14600 : entropy loss: 0.000330044, learning_rate: 0.000551033\n",
            "step 14700 : entropy loss: 0.000531122, learning_rate: 0.000548788\n",
            "step 14800 : entropy loss: 7.18439e-05, learning_rate: 0.000546552\n",
            "step 14900 : entropy loss: 2.52602e-05, learning_rate: 0.000544326\n",
            "step 15000 : entropy loss: 1.29753e-05, learning_rate: 0.000542108\n",
            "step 15100 : entropy loss: 4.09539e-05, learning_rate: 0.0005399\n",
            "step 15200 : entropy loss: 3.79153e-05, learning_rate: 0.0005377\n",
            "step 15300 : entropy loss: 6.82464e-06, learning_rate: 0.00053551\n",
            "step 15400 : entropy loss: 2.00981e-06, learning_rate: 0.000533328\n",
            "step 15500 : entropy loss: 2.73691e-05, learning_rate: 0.000531156\n",
            "step 15600 : entropy loss: 1.16343e-06, learning_rate: 0.000528992\n",
            "step 15700 : entropy loss: 0.000297678, learning_rate: 0.000526837\n",
            "step 15800 : entropy loss: 0.00977153, learning_rate: 0.00052469\n",
            "step 15900 : entropy loss: 0.000315084, learning_rate: 0.000522553\n",
            "step 16000 : entropy loss: 1.31091e-05, learning_rate: 0.000520424\n",
            "step 16100 : entropy loss: 0.000316228, learning_rate: 0.000518304\n",
            "step 16200 : entropy loss: 0.000161886, learning_rate: 0.000516192\n",
            "step 16300 : entropy loss: 7.01317e-05, learning_rate: 0.000514089\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "step 16400 : entropy loss: 8.92064e-06, learning_rate: 0.000511995\n",
            "step 16500 : entropy loss: 1.4698e-06, learning_rate: 0.000509909\n",
            "step 16600 : entropy loss: 4.97174e-06, learning_rate: 0.000507832\n",
            "step 16700 : entropy loss: 0.00156107, learning_rate: 0.000505763\n",
            "step 16800 : entropy loss: 9.02817e-05, learning_rate: 0.000503703\n",
            "step 16900 : entropy loss: 0.00976786, learning_rate: 0.000501651\n",
            "step 17000 : entropy loss: 4.26767e-07, learning_rate: 0.000499607\n",
            "step 17100 : entropy loss: 2.44009e-06, learning_rate: 0.000497572\n",
            "step 17200 : entropy loss: 0.000163324, learning_rate: 0.000495545\n",
            "step 17300 : entropy loss: 2.27614e-05, learning_rate: 0.000493526\n",
            "step 17400 : entropy loss: 1.66172e-06, learning_rate: 0.000491515\n",
            "step 17500 : entropy loss: 0.00129609, learning_rate: 0.000489513\n",
            "step 17600 : entropy loss: 0.0215612, learning_rate: 0.000487519\n",
            "step 17700 : entropy loss: 4.3276e-06, learning_rate: 0.000485533\n",
            "step 17800 : entropy loss: 3.84004e-05, learning_rate: 0.000483555\n",
            "step 17900 : entropy loss: 1.71724e-05, learning_rate: 0.000481585\n",
            "step 18000 : entropy loss: 7.15256e-09, learning_rate: 0.000479623\n",
            "step 18100 : entropy loss: 0.00240087, learning_rate: 0.000477669\n",
            "step 18200 : entropy loss: 0.000189598, learning_rate: 0.000475723\n",
            "step 18300 : entropy loss: 0.000139496, learning_rate: 0.000473785\n",
            "step 18400 : entropy loss: 9.53671e-08, learning_rate: 0.000471855\n",
            "step 18500 : entropy loss: 2.08509e-05, learning_rate: 0.000469932\n",
            "step 18600 : entropy loss: 3.06941e-06, learning_rate: 0.000468018\n",
            "step 18700 : entropy loss: 0.000942694, learning_rate: 0.000466111\n",
            "step 18800 : entropy loss: 7.07018e-05, learning_rate: 0.000464212\n",
            "step 18900 : entropy loss: 7.77076e-06, learning_rate: 0.000462321\n",
            "step 19000 : entropy loss: 0.000940807, learning_rate: 0.000460438\n",
            "step 19100 : entropy loss: 0.00131658, learning_rate: 0.000458562\n",
            "step 19200 : entropy loss: 1.36486e-06, learning_rate: 0.000456694\n",
            "step 19300 : entropy loss: 3.2623e-05, learning_rate: 0.000454833\n",
            "step 19400 : entropy loss: 7.41105e-06, learning_rate: 0.00045298\n",
            "step 19500 : entropy loss: 8.45326e-06, learning_rate: 0.000451135\n",
            "step 19600 : entropy loss: 0.000660857, learning_rate: 0.000449297\n",
            "step 19700 : entropy loss: 0.019083, learning_rate: 0.000447467\n",
            "step 19800 : entropy loss: 1.27554e-07, learning_rate: 0.000445644\n",
            "step 19900 : entropy loss: 1.55947e-05, learning_rate: 0.000443828\n",
            "step 20000 : entropy loss: 2.55845e-05, learning_rate: 0.00044202\n",
            "test accuracy 0.9936\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "TseGjjjW0Uv_",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}